package proc

import (
	"bytes"
	"github.com/PuerkitoBio/goquery"
	"github.com/hu17889/go_spider/core/common/page"
	corerequest "github.com/hu17889/go_spider/core/common/request"
	"github.com/hu17889/go_spider/core/pipeline"
	"github.com/hu17889/go_spider/core/spider"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/transform"
	"io/ioutil"
	"net/http"
	"strings"
	"tesou.io/platform/poem-parent/poem-api/common/base"
	"tesou.io/platform/poem-parent/poem-api/module/core/enums"
	"tesou.io/platform/poem-parent/poem-api/module/core/pojo"
	"tesou.io/platform/poem-parent/poem-core/module/core/service"
)

type WYNewsProcesser struct {
	wynewsService    service.WYNewsService
	SpiderUrl string
	cookies []*http.Cookie
}

func GetWYNewsProcesser() *WYNewsProcesser {
	return &WYNewsProcesser{}
}

func (this *WYNewsProcesser) Startup() {
	newSpider := spider.NewSpider(this, "WYNewsProcesser")

	//newSpider = newSpider.AddUrl(this.SpiderUrl, "html")
	base.Log.Info("start URL:,", this.SpiderUrl)
	hder := make(http.Header)
	hder.Set("User-Agent", `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36`)
	hder.Set("Upgrade-Insecure-Requests", `1`)

	req := corerequest.NewRequest(this.SpiderUrl, "html", "wynews", "GET", "", hder, nil, nil,nil)
	newSpider.AddRequest(req)

	newSpider = newSpider.AddPipeline(pipeline.NewPipelineConsole())
	newSpider.SetThreadnum(1).Run()
}

func (this *WYNewsProcesser) Process(p *page.Page) {
	request := p.GetRequest()
	if !p.IsSucc() {
		base.Log.Info("URL:,", request.Url, p.Errormsg())
		return
	}
	//var allData []map[string]interface{}
	zhi_list_slice := make([]interface{}, 0)
	zhi_list_update_slice := make([]interface{}, 0)
	level := enums.GetListLevel("网易新闻")
	p.GetHtmlParser().Find("table tr").Each(func(i int, selection *goquery.Selection) {
		s := selection.Find("td a").First()
		url, boolUrl := s.Attr("href")
		temp_title := s.Text()
		base.Log.Info("URL:,", temp_title)
		//text, _ := GbkToUtf8([]byte(s.Text()))
		//img := selection.Find("img").First()
		//img_src, _ := img.Attr("src")
		if len(temp_title) != 0 {
			if boolUrl {
				zhi := new(pojo.List)
				if len(url)>0{
					temp_id := strings.Split(url, "article/")
					if len(temp_id)>=2 {
						zhi.ListId =strings.Replace(temp_id[1], ".html", "", -1)
					}else{
						zhi.ListId = ""
					}
					//zhi.ListId =strings.Replace(strings.Split(url, "article/")[1] , ".html", "", -1)
				}else{
					zhi.ListId = ""
				}

				zhi.Title = temp_title
				zhi.SUrl = url
				zhi.Image = ""
				if len(zhi.Title) > 0 {
					//设置朝代
					zhi.Type = level
					exist := this.wynewsService.Exist(zhi)
					if !exist {
						zhi_list_slice = append(zhi_list_slice, zhi)
					} else {
						zhi_list_update_slice = append(zhi_list_update_slice, zhi)
					}
				}
			}
		}
	})
	//执行入库
	this.wynewsService.SaveList(zhi_list_slice)
	this.wynewsService.ModifyList(zhi_list_update_slice)

}

func (this *WYNewsProcesser) Finish() {
	base.Log.Info("zhihu表抓取解析完成 \r\n")

}

/**
部分热榜标题需要转码
*/
func GbkToUtf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}